<?php
//  
use QL\QueryList;
use function GuzzleHttp\json_decode;

class collectionModel extends Model
{

    public function __construct()
    {
        parent::__construct("collection");

    }

    private $pagingurlruleArray = [];

    //引入工具类模型方法
    public function library($lib)
    {
        include_once LIB_PATH . "{$lib}.class.php";
    }

    /**
     * 采集列表链接
     * @date: 2019年6月10日 下午2:38:16
     * @param: variable
     * @return:
     * @author: 龚华尧
     */
    public function collectLinkList($id, $page)
    {
        $collectionDetail = $this->findOne($id);

        $pagingurlruleJsonstr = html_entity_decode($collectionDetail['pagingurlrule']);
        $pagingurlruleArray = json_decode(htmlspecialchars_decode($pagingurlruleJsonstr), true);//pagingurlrule
        $this->pagingurlruleArray = $pagingurlruleArray;
        //采集需要的信息
        $colletionStatus = true;
        while ($colletionStatus) {

            if ($page == "" || $page == 1) {
                $html_url = htmlspecialchars_decode(html_entity_decode($collectionDetail["srcurl"]));
            } else {
                //$html_url = str_replace("{$pagenum}",$page,"Hello $html_url!")
                $html_url = htmlspecialchars_decode(html_entity_decode($collectionDetail["pagingurl"]));
                $html_url = str_replace("{pagenum}", $page, $html_url);
            }
            echo "page:{$page}  {$html_url}<br/>";
            //echo $this->common->get_webHtml($html_url);die();
            //var_dump(isset($pagingurlruleArray["rules"]["baseUrl"]));die();
            $data = QueryList::get($html_url)->rules([
                'link' => [$pagingurlruleArray["rules"]["link"], 'href', '', function ($content) {
                    //利用回调函数补全相对链接
                    $pagingurlruleArray = $this->pagingurlruleArray;

                    if (!preg_match('/(http:\/\/)|(https:\/\/)/i', $content)) {
                        $baseUrl = $pagingurlruleArray["rules"]["baseUrl"];
                        return $baseUrl . $content;
                    } else {
                        return $content;
                    }


                }],
                'text' => [$pagingurlruleArray["rules"]["text"], 'text']
            ])->range($pagingurlruleArray["range"])->query()->getData();


            $re_data = $data->all();


            if (count($re_data) == 0) {
                //没有内容则终止采集
                $colletionStatus = false;
            }

            if ($collectionDetail["pagingnum"] != "" && $collectionDetail["pagingnum"] != 0) {
                if ($page >= $collectionDetail["pagingnum"]) {
                    //没有内容则终止采集
                    $colletionStatus = false;
                }

            }

            $page++;

            //var_dump($re_data);die();
            //详情页采集
            $this->collectPages($id, $re_data);

            flush();

        }


    }

    /**
     * 多个页面的采集，并写入数据
     * @date: 2019年6月10日 下午1:50:09
     * @param: variable
     * @return:
     * @author: 龚华尧
     */
    public function collectPages($id, $ListData)
    {
        $collectionDetail = $this->findOne($id);
        $table_id = $collectionDetail["table_id"];

        $tableModel = new tableModel();
        $tableDetail = $tableModel->findOne($table_id);
        $tablename = $tableModel->getTablenameByTableid($table_id);
        $_tableModel = new Model($tablename);

        $logModel = new logModel();

        $filedModel = new filedModel();

        $collectionfiledruleModel = new collectionfiledruleModel();
        $collectionfiledruleList = $collectionfiledruleModel->getfiledruleListBycollectionid($id);

        $myReg = [];
        foreach ($collectionfiledruleList as $k => $v) {


            if ($filedModel->getfiledTypeByname($table_id, $v["filedname"]) == "文本编辑器") {
                $myReg[$v["filedname"]] = [$v["filedrule"], "html"];
            } else {
                $myReg[$v["filedname"]] = [$v["filedrule"], "text"];
            }

        }

        //var_dump($myReg);die();
        //便利所有的页面
        foreach ($ListData as $k => $v) {
            //需要采集的目标页面
            $page = $v["link"];

            //echo $this->common->get_webHtml($page);die();

            //查询日志，是否已经采集，一采集这不入库
            if ($logModel->where("u4='采集' and u2='{$page}' ")->all()) {
                break;
            }
            echo $page . "<br/>";
            flush();

            //采集规则
            $reg = $myReg;
            $rang = $collectionDetail["detailrange"];
            $ql = QueryList::get($page)->rules($reg)->range($rang)->query();
            $data = $ql->getData(function ($item) {
                return $item;
            });

            $re_data = $data->all()[0];

            //2.验证和处理

            $this->helper('input');
            $re_data = deepspecialchars($re_data);
            $re_data = deepslashes($re_data);


            //单独处理字段采集的数据
            foreach ($re_data as $k => $v) {
                //处理时间
                if ($filedModel->getfiledTypeByname($table_id, $k) == "时间框") {
                    $regex = "'\d{4}/\d{1,2}/\d{1,2}'";
                    $regex2 = "'\d{4}-\d{1,2}-\d{1,2}'";
                    //发布时间 2015/2/11 发布时间 2015/2/13
                    preg_match_all($regex, $re_data[$k], $matches);
                    if (count($matches[0]) == 0) {
                        preg_match_all($regex2, $re_data[$k], $matches);
                    }

                    if (count($matches[0]) > 0) {
                        $re_data[$k] = $matches[0][0];
                    }

                }

                //处理文本域
                if ($filedModel->getfiledTypeByname($table_id, $k) == "文本域") {

                    if (strlen(strip_tags(htmlspecialchars_decode($re_data[$k]))) > 250) {
                        $re_data[$k] = substr(strip_tags(htmlspecialchars_decode($re_data[$k])), 0, 200);
                        $re_data[$k] = str_replace(array("\r\n", "\r", "\n"), "", $re_data[$k]);

                    }

                }

                //处理文本编辑器中的图片和文件
                if ($filedModel->getfiledTypeByname($table_id, $k) == "文本编辑器") {
                    $re_data[$k] = html_entity_decode($re_data[$k]);
                    $content = QueryList::html($re_data[$k]);
                    //处理图片
                    $content->find('img')->map(function ($img) {

                        $ext = strrchr($src, "."); // 得到图片的扩展名
                        if ($ext != ".gif" && $ext != ".jpg" && $ext != ".bmp" && $ext != ".png") {
                            //                                 echo "格式不支持！";
                            //                                 return false;
                        } else {
                            if (!preg_match('/(http:\/\/)|(https:\/\/)/i', $src)) {
                                $arr = parse_url($v["link"]);
                                $domain = $arr['host'];
                                $src = "http://" . $domain . $src;
                                $img->attr('src', $src);
                            }

                        }
                        $localSrc = 'public/grabfile/' . md5($this->common->getOrderId()) . $ext;

                        //将文件保存到本地
                        //$this->common->GrabImage($src,$localSrc);
                        //$img->attr('src',$localSrc);


                    });

                    //处理文件
                    $content->find('a')->map(function ($file) {
                        $src = $file->href;
                        $ext = strrchr($src, "."); // 得到图片的扩展名
                        if ($ext != ".gif" && $ext != ".jpg" && $ext != ".bmp" && $ext != ".png" && $ext != ".doc" && $ext != ".docx" && $ext != ".pdf" && $ext != ".xls" && $ext != ".xlsx") {
//                                 echo "格式不支持！";
//                                 return false;
                        } else {
                            if (!preg_match('/(http:\/\/)|(https:\/\/)/i', $src)) {
                                $arr = parse_url($v["link"]);
                                $domain = $arr['host'];
                                $src = "http://" . $domain . $src;
                                $file->attr('href', $src);
                            }

                        }
                        $localSrc = 'public/grabfile/' . md5($this->common->getOrderId()) . $ext;

                        //将文件保存到本地
                        //$this->common->GrabImage($src,$localSrc);
                        //$file->attr('href',$localSrc);
                    });

                }

            }


            //记录日志
            $data_log["u1"] = "system";
            $data_log["u2"] = $page;
            $data_log["u4"] = "采集";
            $logModel->insert($data_log);

            //判定是否采集到数据
            if (count($re_data) == 0) {
                continue;
            }
            //添加额外设定的参数
            $table_valuesJsonstr = html_entity_decode($collectionDetail['table_values']);
            $table_valuesArray = json_decode(htmlspecialchars_decode($table_valuesJsonstr), true);//pagingurlrule
            if (count($table_valuesArray) > 0) {
                $re_data = array_merge($table_valuesArray, $re_data);
            }

            //添加来源
            foreach ($_tableModel->getFieldsAndTypes() as $k => $v) {
                if ($v["column_name"] == "url") {
                    $re_data["url"] = $page;
                }
                if ($v["column_name"] == "dtime") {
                    $re_data["dtime"] = date("Y-m-d H:m:s");
                }
            }

            //3调用模型完成入库并给出提示
            if ($_tableModel->insert($re_data)) {
                $data_return["status"] = true;
                $data_return["msg"] = "添加成功";
                $data_return["code"] = 0;

            } else {
                $data_return["msg"] = "添加失败";

            }
        }

    }

    /**
     * 测试列表链接
     * @date: 2019年6月10日 下午2:38:16
     * @param: variable
     * @return:
     * @author: 龚华尧
     */
    public function ShowcollectLinkListData($id, $page, $step)
    {
        $collectionDetail = $this->findOne($id);

        $pagingurlruleJsonstr = html_entity_decode($collectionDetail['pagingurlrule']);
        $pagingurlruleArray = json_decode(htmlspecialchars_decode($pagingurlruleJsonstr), true);//pagingurlrule
        $this->pagingurlruleArray = $pagingurlruleArray;
        //采集需要的信息
        $colletionStatus = true;
        while ($colletionStatus) {

            if ($page == "" || $page == 1) {
                $html_url = htmlspecialchars_decode(html_entity_decode($collectionDetail["srcurl"]));
            } else {
                //$html_url = str_replace("{$pagenum}",$page,"Hello $html_url!")
                $html_url = htmlspecialchars_decode(html_entity_decode($collectionDetail["pagingurl"]));
                $html_url = str_replace("{pagenum}", $page, $html_url);
            }
            echo "<p><font size='6' color='red'>当前采集列表页面：</font><p>";
            echo $html_url;
            //查询首页数据
            if ($step == "1") {
                echo "<p><font size='5' color='red'>当前采集首页HTML程序：</font><p>";
                echo $this->common->get_webHtml($html_url);
                die();
            }
            $data = QueryList::get($html_url)->rules([
                'link' => [$pagingurlruleArray["rules"]["link"], 'href', '', function ($content) {
                    //利用回调函数补全相对链接
                    $pagingurlruleArray = $this->pagingurlruleArray;

                    if (!preg_match('/(http:\/\/)|(https:\/\/)/i', $content)) {
                        $baseUrl = $pagingurlruleArray["rules"]["baseUrl"];
                        return $baseUrl . $content;
                    } else {
                        return $content;
                    }
                }],
                [
                    'headers' => [
                        'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.90 Mobile Safari/537.36 ',
                    ]
                ],
                'text' => [$pagingurlruleArray["rules"]["text"], 'text']
            ])->range($pagingurlruleArray["range"])->query()->getData();

            $re_data = $data->all();

            //查询列表数据数据
            if ($step == "2") {
                foreach ($re_data as $k1 => $v1) {
                    echo "<p><font size='4' color='red'>列表链接：</font><p>";
                    echo $v1["link"] . "<p>";
                    echo "<p><font size='4' color='red'>列表标题：</font><p>";
                    echo $v1["text"] . "<p>";;

                }
                die();
            }

            if (count($re_data) == 0) {
                //没有内容则终止采集
                $colletionStatus = false;
            }

            if ($collectionDetail["pagingnum"] != "" && $collectionDetail["pagingnum"] != 0) {
                if ($page >= $collectionDetail["pagingnum"]) {
                    //没有内容则终止采集
                    $colletionStatus = false;
                }
            }
            $page++;
            //测试详情页采集
            $this->ShowcollectPages($id, $re_data, $step);
            flush();
        }

    }

    /**
     * 多个页面的采集，并写入数据
     * @date: 2019年6月10日 下午1:50:09
     * @param: variable
     * @return:
     * @author: 龚华尧
     */
    public function ShowcollectPages($id, $ListData, $step)
    {
        $collectionDetail = $this->findOne($id);
        $table_id = $collectionDetail["table_id"];

        $tableModel = new tableModel();
        $tableDetail = $tableModel->findOne($table_id);
        $tablename = $tableModel->getTablenameByTableid($table_id);
        $_tableModel = new Model($tablename);

        $logModel = new logModel();

        $filedModel = new filedModel();

        $collectionfiledruleModel = new collectionfiledruleModel();
        $collectionfiledruleList = $collectionfiledruleModel->getfiledruleListBycollectionid($id);

        $myReg = [];
        foreach ($collectionfiledruleList as $k => $v) {


            if ($filedModel->getfiledTypeByname($table_id, $v["filedname"]) == "文本编辑器") {
                $myReg[$v["filedname"]] = [$v["filedrule"], "html"];
            } else {
                $myReg[$v["filedname"]] = [$v["filedrule"], "text"];
            }

        }

        //var_dump($myReg);die();
        //便利所有的页面
        foreach ($ListData as $k => $v) {
            //需要采集的目标页面
            $page = $v["link"];

            //echo $this->common->get_webHtml($page);die();

            //查询日志，是否已经采集，一采集这不入库
            echo "<p><font size='6' color='red'>当前采集详情页面地址:</font><p>";
            echo $page;
//             echo $page."<br/>";flush();

            //测试采集内容区域
            if ($step == 3) {
                //采集规则
                $reg = [
                    //采集文章正文内容,利用过滤功能去掉文章中的超链接，但保留超链接的文字，并去掉版权、JS代码等无用信息
                    'content' => [$collectionDetail["detailrange"], 'html', 'a -script']
                ];
                $rang = $collectionDetail["detailrange"];
                $ql = QueryList::get($page)->rules($reg)->query();
                $data = $ql->getData(function ($item) {
                    return $item;
                });

                $re_data = $data->all()[0];
                if (count($re_data) > 0) {
                    echo "<p><font size='3' color='red'>详情页内容：：</font><p>";
                    echo $re_data["content"];
                } else {
                    echo "该区域没有内容，请重新设置";
                }
                die();
            }
            //采集规则
            $reg = $myReg;
            $rang = $collectionDetail["detailrange"];
            $ql = QueryList::get($page)->rules($reg)->range($rang)->query();
            $data = $ql->getData(function ($item) {
                return $item;
            });

            $re_data = $data->all()[0];
            if (count($re_data) == 0) {
                echo "该区域没有内容，请重新设置";
                die();
            }
            //2.验证和处理

            $this->helper('input');
            $re_data = deepspecialchars($re_data);
            $re_data = deepslashes($re_data);


            //单独处理字段采集的数据
            foreach ($re_data as $k => $v) {
                //处理时间
                if ($filedModel->getfiledTypeByname($table_id, $k) == "时间框") {
                    $regex = "'\d{4}/\d{1,2}/\d{1,2}'";
                    $regex2 = "'\d{4}-\d{1,2}-\d{1,2}'";
                    //发布时间 2015/2/11 发布时间 2015/2/13
                    preg_match_all($regex, $re_data[$k], $matches);
                    if (count($matches[0]) == 0) {
                        preg_match_all($regex2, $re_data[$k], $matches);
                    }

                    if (count($matches[0]) > 0) {
                        $re_data[$k] = $matches[0][0];
                    }

                }

                //处理文本域
                if ($filedModel->getfiledTypeByname($table_id, $k) == "文本域") {

                    if (strlen(strip_tags(htmlspecialchars_decode($re_data[$k]))) > 250) {
                        $re_data[$k] = substr(strip_tags(htmlspecialchars_decode($re_data[$k])), 0, 200);
                        $re_data[$k] = str_replace(array("\r\n", "\r", "\n"), "", $re_data[$k]);

                    }

                }


            }


            //判定是否采集到数据
            if (count($re_data) == 0) {
                echo "没有采集到数据，请检查字段采集配置";
                die();
            }
            //添加额外设定的参数
            $table_valuesJsonstr = html_entity_decode($collectionDetail['table_values']);
            $table_valuesArray = json_decode(htmlspecialchars_decode($table_valuesJsonstr), true);//pagingurlrule
            if (count($table_valuesArray) > 0) {
                $re_data = array_merge($table_valuesArray, $re_data);
            }

            //添加来源
            foreach ($_tableModel->getFieldsAndTypes() as $k => $v) {
                if ($v["column_name"] == "url") {
                    $re_data["url"] = $page;
                }
                if ($v["column_name"] == "dtime") {
                    $re_data["dtime"] = date("Y-m-d H:m:s");
                }
                //var_dump($v);
            }

//                                 var_dump($re_data);
//                                 die();

            //测试采集字段
            if ($step == 4) {
                $filedModel = new filedModel();
                $filedList = $filedModel->getallFiledByTableid($table_id);
                //挂载  组件类
                $this->library('Component');
                $component = new Component();
                echo "采集内容如下：<p><p>";
                foreach ($filedList as $v) {
                    if ($re_data[$v['u1']] == "") {
                        continue;
                    }
                    echo "<p><font size='3' color='red'>{$v["u2"]}：</font><p>";
                    echo $component->showKj($v['u7'], $v['u2'], $v['u1'], $re_data[$v['u1']], $v['u3'], $v['id']);
                    echo "<p>";
                }
                die();
            }


        }


    }

}